{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " '/Users/wuzhong/gitee/text-classify-course/notebook',\n",
       " '/usr/local/Cellar/python/3.7.1/Frameworks/Python.framework/Versions/3.7/lib/python37.zip',\n",
       " '/usr/local/Cellar/python/3.7.1/Frameworks/Python.framework/Versions/3.7/lib/python3.7',\n",
       " '/usr/local/Cellar/python/3.7.1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/lib-dynload',\n",
       " '/usr/local/lib/python3.7/site-packages',\n",
       " '/usr/local/lib/python3.7/site-packages/IPython/extensions',\n",
       " '/Users/wuzhong/.ipython',\n",
       " '../',\n",
       " '/Users/wuzhong/gitee/text-classify-course']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import sys\n",
    "sys.path.append(\"../\")\n",
    "sys.path.append(\"/Users/wuzhong/gitee/text-classify-course\")\n",
    "sys.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /var/folders/7w/2zfqy3d94l5fc7ckfvzphnbw0000gp/T/jieba.cache\n",
      "Loading model cost 0.941 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "import src.core.PandasUtils as pandasutils\n",
    "import src.core.tf_idf as tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_model(lr, path):\n",
    "    from sklearn.externals import joblib\n",
    "    joblib.dump(lr, path)\n",
    "\n",
    "\n",
    "def load_model(path):\n",
    "    from sklearn.externals import joblib\n",
    "    return joblib.load(path)\n",
    "\n",
    "\n",
    "def model_exist(path):\n",
    "    import os\n",
    "    return os.path.exists(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"../data/training.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.columns = [\"type\",\"text\"]\n",
    "data[\"type\"] = data[\"type\"]. map(lambda s: int(s) - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>公司的主营业务为向中小微企业、个体工商户、农户等客户提供贷款服务，自设立以来主营业务未发生过变化。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>公司立足于商业地产服务，致力于为商业地产开发、销售、运营全产业链提供一整套增值服务，业务覆盖...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   type                                               text\n",
       "0     1  公司的主营业务为向中小微企业、个体工商户、农户等客户提供贷款服务，自设立以来主营业务未发生过变化。\n",
       "1     0  公司立足于商业地产服务，致力于为商业地产开发、销售、运营全产业链提供一整套增值服务，业务覆盖..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start data conversion\n",
      "Index(['C27', 'C36', 'C3660', 'G58', 'GMP', 'IT', '一次性', '一汽', '三农', '三方',\n",
      "       ...\n",
      "       '饲养', '饲料', '香菇', '高分', '高分子', '鸡蛋', '鸭', '龙头', '龙头企业', ''],\n",
      "      dtype='object', length=500)\n",
      "   C27  C36  C3660  G58  GMP   IT  一次性   一汽   三农   三方 ...    饲养   饲料   香菇  \\\n",
      "0  0.0  0.0    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0 ...   0.0  0.0  0.0   \n",
      "1  0.0  0.0    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0 ...   0.0  0.0  0.0   \n",
      "\n",
      "    高分  高分子   鸡蛋    鸭   龙头  龙头企业      \n",
      "0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  \n",
      "1  0.0  0.0  0.0  0.0  0.0   0.0  0.0  \n",
      "\n",
      "[2 rows x 500 columns]\n"
     ]
    }
   ],
   "source": [
    "path = '../dist/train_data_full.pkl'\n",
    "if model_exist(path):\n",
    "    print(\"load model from \", path)\n",
    "    df = load_model(path)\n",
    "else:\n",
    "    print(\"start data conversion\")\n",
    "    df = pandasutils.convert2KeywordsDataframs(data.values)\n",
    "    save_model(df, path)\n",
    "    \n",
    "print(df[\"X\"].head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
